# -*- coding: utf-8 -*-
"""
@auth:sirius
@time:2017.10.24
"""
from sklearn.datasets import fetch_20newsgroups
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import classification_report

'''
数据集
'''
news = fetch_20newsgroups(data_home='./datasets', subset='all')
print '总的样本量：', len(news.data)
print news.target_names

category = ['alt.atheism', 'comp.graphics', 'soc.religion.christian', 'sci.med']
news_train = fetch_20newsgroups(data_home='./datasets', subset='train', categories=category)
print '总的样本量：', len(news_train.data)
print news_train.target_names

print '训练样本的分布情况：'
print news_train.target_names
print np.bincount(news_train.target)

print '.\n'.join(news_train.data[0].split('.\n')[:3]) # 新闻内容

print news_train.data[0:10] # 前10条新闻
print news_train.target[0:10] # 前10条新闻类别标签

for t in news_train.target[0:10]:
    print t, '<-->', news_train.target_names[t]

'''
特征向量的抽取
'''
vec = CountVectorizer()
X_train = vec.fit_transform(news_train.data)
print X_train.shape

'''
训练分类器
'''
mnb = MultinomialNB(alpha=1.0) # alpha附加拉普拉斯平滑参数
mnb.fit(X_train, news_train.target)

'''
计算分类器在训练样本集上的误差
'''
predicted_y_train = mnb.predict(X_train)
error_predicted = predicted_y_train != news_train.target
error_count = sum(error_predicted)
print '分类器在训练样本集上的错分个数：', error_count
print '分类器在训练集上的正确率：', 1-error_count/float(len(news_train.target))
error_predicted_samples = news_train.target[error_predicted]
for t in error_predicted_samples:
    print t, '<-->', news_train.target_names[t]

'''
加载测试集，计算测试误差
'''
category = ['alt.atheism', 'comp.graphics', 'soc.religion.christian', 'sci.med']
news_test = fetch_20newsgroups(data_home='./datasets', subset='test', categories=category)
print '总的测试样本数量：', len(news_test.data)
print news_test.target_names

print '测试样本的分布情况：'
print news_test.target_names
print np.bincount(news_test.target)

'''
提取测试集的特征向量集合
'''
X_test = vec.transform(news_test.data)
print X_test.shape

'''
对测试特征向量集合进行预测，计算预测误差
'''
predicted_y_test = mnb.predict(X_test)
correct_predicted = predicted_y_test == news_test.target
accuracy = np.mean(correct_predicted)
print '分类器在测试集上的正确率：', accuracy

'''
评估分类器性能
'''
print classification_report(news_test.target, predicted_y_test, target_names=news_test.target_names)

error_predicted_samples = news_test.target[predicted_y_test != news_test.target]
print '错分的测试样本数量：', len(error_predicted_samples)
for t in error_predicted_samples:
    print t, '<-->', news_test.target_names[t]

print news_test.target_names # 每个类的错分样本数量
print np.bincount(error_predicted_samples)